{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from typing import  List, Dict, Union\n",
    "\n",
    "from datasets import load_dataset\n",
    "import sys\n",
    "sys.path.append(\"/home/jie/gitee/pku_industry/general\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = \"/home/jie/gitee/pku_industry/LLM_second_cls/output/氢能产业链企业_9k.csv\"\n",
    "df = pd.read_csv(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset({\n",
      "    features: ['企业名称', '英文名称', '统一社会信用代码', '企业类型', '经营状态', '成立日期', '核准日期', '法定代表人', '注册资本', '实缴资本', '参保人数', '公司规模', '经营范围', '注册地址', '营业期限', '纳税人识别号', '工商注册号', '组织机构代码', '纳税人资质', '曾用名', '所属省份', '所属城市', '所属区县', '网站链接', '所属行业', '一级行业分类', '二级行业分类', '三级行业分类', '登记机关', '经度', '纬度', '网址', '氢能产业链节点', '上链原因'],\n",
      "    num_rows: 9200\n",
      "})\n"
     ]
    }
   ],
   "source": [
    "dataset = load_dataset(\"csv\", data_files=f, split=\"train\")\n",
    "print(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_address(item):\n",
    "    # 所属省份\n",
    "    prov = item[\"所属省份\"]\n",
    "    for p in [\"北京\", \"天津\", \"河北\"]:\n",
    "        if p in prov:\n",
    "            return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16946716dc30483986c70a1e7d5c8387",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating CSV from Arrow format:   0%|          | 0/2 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset.filter(filter_address).to_csv(\"京津冀.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.filter(filter_address)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-            73\n",
       "500万人民币      15\n",
       "1000万人民币     13\n",
       "100万人民币      11\n",
       "50万人民币       10\n",
       "             ..\n",
       "190万人民币       1\n",
       "22000万人民币     1\n",
       "167300万日元     1\n",
       "42万人民币        1\n",
       "420万人民币       1\n",
       "Name: count, Length: 123, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(dataset['实缴资本']).value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "def trans_money(name):\n",
    "\n",
    "    def func(item):\n",
    "        s = item[name]\n",
    "        item[f\"{name}_money\"] = 0.0\n",
    "        if not isinstance(s, str):\n",
    "            return item\n",
    "        s = s.strip()\n",
    "        if len(s) < 4:\n",
    "            return item\n",
    "\n",
    "        exchange_rates = {\n",
    "            \"人民币\": 1.00,  # CNY\n",
    "            \"美元\": 7.12,  # USD\n",
    "            \"欧元\": 7.84,  # EUR\n",
    "            \"日元\": 0.04993,  # JPY\n",
    "            \"英镑\": 9.28,  # GBP\n",
    "            \"澳大利亚元\": 4.76,  # AUD\n",
    "            \"加元\": 5.24,  # CAD\n",
    "            \"港元\": 0.91,  # HKD\n",
    "            \"港币\": 0.91,  # HKD\n",
    "            \"新加坡元\": 5.45,  # SGD\n",
    "            \"瑞士法郎\": 8.35,  # CHF\n",
    "            \"韩元\": 0.00053,  # KRW\n",
    "            \"泰铢\": 0.21,  # THB\n",
    "            \"新台币\": 0.22,  # TWD\n",
    "        }\n",
    "        money = 0\n",
    "        try:\n",
    "            idx = s.index(\"万\")\n",
    "            if idx == -1:\n",
    "                return item\n",
    "            money = s[:idx]\n",
    "            bi = s[idx + 1 :]\n",
    "            assert bi in exchange_rates.keys()\n",
    "            money = float(money) * exchange_rates[bi]\n",
    "        except Exception as e:\n",
    "            print(e, s)\n",
    "        item[f\"{name}_money\"] = money\n",
    "        return item\n",
    "\n",
    "    return func"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['企业名称', '英文名称', '统一社会信用代码', '企业类型', '经营状态', '成立日期', '核准日期', '法定代表人', '注册资本', '实缴资本', '参保人数', '公司规模', '经营范围', '注册地址', '营业期限', '纳税人识别号', '工商注册号', '组织机构代码', '纳税人资质', '曾用名', '所属省份', '所属城市', '所属区县', '网站链接', '所属行业', '一级行业分类', '二级行业分类', '三级行业分类', '登记机关', '经度', '纬度', '网址', '氢能产业链节点', '上链原因'],\n",
       "    num_rows: 1081\n",
       "})"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset.map(trans_money(\"实缴资本\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[\"实缴资本_money\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 首先按照地区的市排序，再按照公司规模排序"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.0009, 0.0134, 0.0178, 0.0267, 0.0890, 0.4452, 0.8903])"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "\n",
    "def l2_normalization_torch(data):\n",
    "    norm = torch.norm(data)\n",
    "    return data / norm\n",
    "\n",
    "\n",
    "data = torch.tensor([1, 15, 20, 30, 100, 500, 1000], dtype=torch.float32)\n",
    "normalized_data = l2_normalization_torch(data)\n",
    "normalized_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1123.1768)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.norm(torch.tensor([1, 15, 20, 30, 100, 500, 1000], dtype=torch.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.7311, 1.0000, 1.0000, 1.0000, 1.0000])"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def sigmoid_scaling_torch(data):\n",
    "    return 1 / (1 + torch.exp(-data))\n",
    "\n",
    "data = torch.tensor([1, 10, 100, 1000, 10000], dtype=torch.float32)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.5002, 0.5033, 0.5045, 0.5067, 0.5222, 0.6095, 0.7090])"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaled_data = sigmoid_scaling_torch(normalized_data)\n",
    "scaled_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# 创建一个随机张量\n",
    "x = torch.rand([1, 15, 20, 30, 100, 500, 1000])\n",
    "\n",
    "# 对每个向量进行归一化，使其 L2 范数为 1\n",
    "x_normalized = F.normalize(x, p=2, dim=1)\n",
    "print(x_normalized)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
